Accueil Explorer Aide
Inscription Connexion
J1X
/
huginn
Suivre 2
Voter 0
Embranchement 0

Parcourir la Source
WebsiteAgent: Provide a variable _response_ for interpolation.

* `_response_.headers.Header-Name` is expanded to the value of header
  named _Header-Name_.

* `_response_.status` is expanded to the HTTP status as integer, which
  is almost always 200, since currently the WebsiteAgent will only
  create an event when a request succeeds, and the request method is
  limited to GET.

Akinori MUSHA 11 ans auparavant
Parent
e2f2ff5da8
Commettre
c21bada226
2 fichiers modifiés avec 64 ajouts et 5 suppressions
Afficher les stats Diff
  1. 32 4
      app/models/agents/website_agent.rb
  2. 32 1
      spec/models/agents/website_agent_spec.rb

+ 32 - 4
app/models/agents/website_agent.rb
Voir le fichier 

@@ -75,6 +75,14 @@ module Agents
75 75 
       The `headers` field is optional.  When present, it should be a hash of headers to send with the request.
76 76
77 77 
       The WebsiteAgent can also scrape based on incoming events. It will scrape the url contained in the `url` key of the incoming event payload.
78 
+
79 
+      In Liquid templating, the following variable is available:
80 
+
81 
+      * `_response_`: A response object with the following keys:
82 
+
83 
+          * `status`: HTTP status as integer. (Almost always 200)
84 
+
85 
+          * `headers`: Reponse headers; for example, `{{ _response_.headers.Content-Type }}` expands to the value of the Content-Type header.  Keys are insentitive to cases and -/_.
78 86 
     MD
79 87
80 88 
     event_description do
 
@@ -149,7 +157,10 @@ module Agents
149 157 
       Array(in_url).each do |url|
150 158 
         log "Fetching #{url}"
151 159 
         response = faraday.get(url)
152 
-        if response.success?
160 
+        raise "Failed: #{response.inspect}" unless response.success?
161 
+
162 
+        interpolation_context.stack {
163 
+          interpolation_context['_response_'] = ResponseDrop.new(response)
153 164 
           body = response.body
154 165 
           if (encoding = interpolated['force_encoding']).present?
155 166 
             body = body.encode(Encoding::UTF_8, encoding)
 
@@ -195,9 +206,7 @@ module Agents
195 206 
               create_event :payload => result
196 207 
             end
197 208 
           end
198 
-        else
199 
-          raise "Failed: #{response.inspect}"
200 
-        end
209 
+        }
201 210 
       end
202 211 
     rescue => e
203 212 
       error e.message
 
@@ -344,5 +353,24 @@ module Agents
344 353 
     rescue
345 354 
       false
346 355 
     end
356 
+
357 
+    # Wraps Faraday::Response
358 
+    class ResponseDrop < LiquidDroppable::Drop
359 
+      def headers
360 
+        HeaderDrop.new(@object.headers)
361 
+      end
362 
+
363 
+      # Integer value of HTTP status
364 
+      def status
365 
+        @object.status
366 
+      end
367 
+    end
368 
+
369 
+    # Wraps Faraday::Utilsa::Headers
370 
+    class HeaderDrop < LiquidDroppable::Drop
371 
+      def before_method(name)
372 
+        @object[name.tr('_', '-')]
373 
+      end
374 
+    end
347 375 
   end
348 376 
 end

+ 32 - 1
spec/models/agents/website_agent_spec.rb
Voir le fichier 

@@ -3,7 +3,11 @@ require 'spec_helper'
3 3 
 describe Agents::WebsiteAgent do
4 4 
   describe "checking without basic auth" do
5 5 
     before do
6 
-      stub_request(:any, /xkcd/).to_return(:body => File.read(Rails.root.join("spec/data_fixtures/xkcd.html")), :status => 200)
6 
+      stub_request(:any, /xkcd/).to_return(body: File.read(Rails.root.join("spec/data_fixtures/xkcd.html")),
7 
+                                           status: 200,
8 
+                                           headers: {
9 
+                                             'X-Status-Message' => 'OK'
10 
+                                           })
7 11 
       @valid_options = {
8 12 
         'name' => "XKCD",
9 13 
         'expected_update_period_in_days' => "2",
 
@@ -305,6 +309,17 @@ describe Agents::WebsiteAgent do
305 309 
         event.payload['slogan'].should == "A webcomic of romance, sarcasm, math, and language."
306 310 
       end
307 311
312 
+      it "should interpolate _response_" do
313 
+        @valid_options['extract']['response_info'] =
314 
+          @valid_options['extract']['url'].merge(
315 
+            'value' => '"{{ "The reponse was " | append:_response_.status | append:" " | append:_response_.headers.X-Status-Message | append:"." }}"'
316 
+          )
317 
+        @checker.options = @valid_options
318 
+        @checker.check
319 
+        event = Event.last
320 
+        event.payload['response_info'].should == 'The reponse was 200 OK.'
321 
+      end
322 
+
308 323 
       describe "JSON" do
309 324 
         it "works with paths" do
310 325 
           json = {
 
@@ -490,6 +505,22 @@ fire: hot
490 505 
           'to' => 'http://dynamic.xkcd.com/random/comic/',
491 506 
         }
492 507 
       end
508 
+
509 
+      it "should interpolate values from incoming event payload and _response_" do
510 
+        @event.payload['title'] = 'XKCD'
511 
+
512 
+        lambda {
513 
+          @valid_options['extract'] = {
514 
+            'response_info' => @valid_options['extract']['url'].merge(
515 
+              'value' => '{% capture sentence %}The reponse from {{title}} was {{_response_.status}} {{_response_.headers.X-Status-Message}}.{% endcapture %}{{sentence | to_xpath}}'
516 
+            )
517 
+          }
518 
+          @checker.options = @valid_options
519 
+          @checker.receive([@event])
520 
+        }.should change { Event.count }.by(1)
521 
+
522 
+        Event.last.payload['response_info'].should == 'The reponse from XKCD was 200 OK.'
523 
+      end
493 524 
     end
494 525 
   end
495 526